﻿using System;
using System.Collections;
using System.Collections.Generic;
using System.Text;
using System.Threading;
using System.Drawing;
using System.Text.RegularExpressions;
using com.crawler.common;
using System.Net.Sockets;
using com.crawler.db.Interface;

namespace com.crawler.components
{
 

    public class CrawlerManager
    {
        /// <summary>
        /// 系统配置信息
        /// </summary>
        private SysConfigInfo m_sys_config = null;

        BaiDuChapter m_baidu_mgr = new BaiDuChapter();

        /// <summary>
        /// 收录地址堆栈
        /// </summary>
        private Queue<SiteUrl> m_queue_url_list = new Queue<SiteUrl>();

        /// <summary>
        /// 新书Url队列
        /// </summary>
        private Queue<SiteUrl> m_new_book_url_queue = new Queue<SiteUrl>();

        /// <summary>
        /// 重点小说Url队列
        /// </summary>
        private Queue<SiteUrl> m_important_book_url_queue = new Queue<SiteUrl>();

        /// <summary>
        /// 更新小说Url队列
        /// </summary>
        private Queue<SiteUrl> m_update_book_url_queue = new Queue<SiteUrl>();

        /// <summary>
        /// 站点列表
        /// </summary>
        List<BookSiteInfo> m_list = null;

        /// <summary>
        /// 数据处理对象
        /// </summary>
        DBManager m_db_mgr = new DBManager();

        /// <summary>
        /// 运行中的线程集合
        /// </summary>
        Thread[] m_thd_list = null;

        static object m_obj = new object();

    

        /// <summary>
        /// 收录类型
        /// </summary>
        public Consts.CrawlerType CrawlerType { get; set; }

        LuceneManager m_lucene_mgr = new LuceneManager();


        public CrawlerManager()
        {
            try
            {
                m_sys_config = SerializationHelper.Load(typeof(SysConfigInfo), Consts.SysConfigFileName) as SysConfigInfo;

                m_list = SerializationHelper.Load(typeof(List<BookSiteInfo>), Consts.SiteFileName) as List<BookSiteInfo>;

            }
            catch (Exception ex)
            {
                LogHelper.Error("加载系统配置失败" + ex.ToString());
            }
        }

        /// <summary>
        /// 获取收录小说的地址
        /// </summary>
        /// <returns></returns>
        private void GetCrawlerUrl(Consts.CrawlerType i_crawler_type,int i_page_index)
        {
       
            lock (m_obj)
            {
                //数据库更新所有的小说
                if (i_crawler_type == Consts.CrawlerType.UpdateBook && m_db_mgr != null)
                {
                    m_update_book_url_queue = m_db_mgr.GetUpdateBook(m_list, i_page_index);
                }//更新重点小说
                else if (i_crawler_type == Consts.CrawlerType.ImportantBook && m_db_mgr != null)
                {
                    List<BookInfo> t_list = m_db_mgr.GetUpdateBookList();

                    if (t_list != null && t_list.Count > 0)
                    {

                        List<BookSiteInfo> t_temp_list = null;
                        foreach (BookInfo t_book in t_list)
                        {
                            SiteUrl t_site = new SiteUrl();
                            t_site.Url = t_book.ChapterListUrl;
                            t_site.BookType = Consts.BookPageType.ChapterList;
                            t_site.SiteId = t_book.SiteId;
                            t_site.BookId = t_book.BookId;
                            t_site.BookName = t_book.BookName;
                            t_site.LastUpdateChapterId = t_book.LastUpdateChapterId;
                            t_site.LastUpdateChapterName = t_book.LastUpdateChapterName;
                            t_site.SiteName = t_book.SiteName;
                            t_temp_list = m_list.FindAll(delegate(BookSiteInfo t_site_info)
                            {
                                return t_site_info.SiteId == t_book.SiteId;
                            });
                            if (t_temp_list != null && t_temp_list.Count > 0)
                            {
                                t_site.ChapterOrder = t_temp_list[0].ChapterOrder;
                            }

                            m_important_book_url_queue.Enqueue(t_site);
                        }
                  
                    }
                }//新书收录
                else if (i_crawler_type == Consts.CrawlerType.NewBook)
                {
                    if (m_list != null && m_list.Count > 0)
                    {

                        foreach (BookSiteInfo t_site_info in m_list)
                        {
                            if (t_site_info.IsForbid)
                                continue;

                            for (int index = t_site_info.StartPageIndex; index <= t_site_info.EndPageIndex; index++)
                            {

                                string t_book_list_url = t_site_info.BookListUrl;
                          
                                SiteUrl t_site = new SiteUrl();
                                t_site.Url = string.Format(t_book_list_url, index);
                                LogHelper.Info(t_site.Url);
                                t_site.BookType = Consts.BookPageType.BookList;
                                t_site.SiteId = t_site_info.SiteId;
                                t_site.ChapterOrder = t_site_info.ChapterOrder;
                                t_site.SiteName = t_site_info.SiteName;
                                m_new_book_url_queue.Enqueue(t_site);
                            }

                        }


                    }

                   
                }
     
            }
        }

  


        /// <summary>
        /// 触发发送消息的事件
        /// </summary>
        public event SendMsgEventHandler SendMsg;

        #region 消息回送
        /// <summary>
        /// 发送消息到客户端
        /// </summary>
        /// <param name="i_msg"></param>
        private void SendMsg2Client(object i_msg, MsgType i_msg_type)
        {
            SendMsgEventArgs e = new SendMsgEventArgs();
            e.Msg = i_msg;
            e.MsgType = i_msg_type;
            if (SendMsg != null)
            {
                SendMsg(this, e);
            }

        }

        /// <summary>
        /// 停止应用
        /// </summary>
        public void StopApp()
        {
            if (m_thd_list != null)
            {
                for (int index = 0; index < m_thd_list.Length; index++)
                {
                    m_thd_list[index].Abort();
                    //线程运行提示
                    ThreadMsg t_thd_msg = new ThreadMsg();

                    t_thd_msg.ThreadId = m_thd_list[index].Name.ToString();
                    t_thd_msg.Action = "线程已经停止";
                    t_thd_msg.ImageIndex = 3;
                    t_thd_msg.BackColor = Color.WhiteSmoke;
                    t_thd_msg.ForeColor = Color.Red;
                    SendMsg2Client(t_thd_msg, MsgType.Thread);

                }
            }
        }

     

        /// <summary>
        /// 开始应用
        /// </summary>
        public void StartApp()
        {
         
            Thread t_thd = new Thread(new ThreadStart(InitQueue));
            t_thd.Start();

            if ( CrawlerType == Consts.CrawlerType.ALL)
            {
                m_thd_list = new Thread[8];
                //两个线程收录新书
                for (int index = 0; index < m_thd_list.Length; index++)
                {
                    m_thd_list[index] = new Thread(new ThreadStart(StartNewBookCrawler));
                    m_thd_list[index].Name = index.ToString();
                    m_thd_list[index].Start();
                    LogHelper.Info("启动新书线程");
                }

                //两个线程收录重点小说
                //for (int index = 1; index < 3; index++)
                //{
                //    m_thd_list[index] = new Thread(new ThreadStart(StartImportantBook));
                //    m_thd_list[index].Name = index.ToString();
                //    m_thd_list[index].Start();
                //    LogHelper.Info("启动重点收录线程");
                //}

                //BaiDuManager t_baidu_mgr = new BaiDuManager();
               // t_baidu_mgr.StartSEOBook();

                //六个线程收录更新小说
                //for (int index = 3; index < 5; index++)
                //{
                //    m_thd_list[index] = new Thread(new ThreadStart(StartUpdateBookCrawler));
                //    m_thd_list[index].Name = index.ToString();
                //    m_thd_list[index].Start();
                //    LogHelper.Info("启动更新收录线程");
                //}
            }
            else
            {
                m_thd_list = new Thread[m_sys_config.ThreadCount];
                for (int index = 0; index < m_sys_config.ThreadCount; index++)
                {
                    if (CrawlerType == Consts.CrawlerType.ImportantBook)
                    {
                        m_thd_list[index] = new Thread(new ThreadStart(StartImportantBook));
                    }
                    else if (CrawlerType == Consts.CrawlerType.NewBook)
                    {
                        m_thd_list[index] = new Thread(new ThreadStart(StartNewBookCrawler));
                    }
                    else if (CrawlerType == Consts.CrawlerType.UpdateBook)
                    {
                        m_thd_list[index] = new Thread(new ThreadStart(StartUpdateBookCrawler));
                    }
                    m_thd_list[index].Name = index.ToString();
                    m_thd_list[index].Start();
                }

                //BaiDuManager t_baidu_mgr = new BaiDuManager();
                //t_baidu_mgr.StartSEOBook();
            }
        }

        private void InitQueue()
        {
            int t_page_index = 1;

        
            while (true)
            {
                if (CrawlerType == Consts.CrawlerType.ALL)
                {
                    if (m_new_book_url_queue == null || m_new_book_url_queue.Count <= 0)
                    {
                        GetCrawlerUrl(Consts.CrawlerType.NewBook, 0);
                        //休息30分钟
                        Thread.Sleep(60 * 1000 * 30);
                    }

                    if (m_update_book_url_queue == null || m_update_book_url_queue.Count <= 0)
                    {
                        GetCrawlerUrl(Consts.CrawlerType.UpdateBook, t_page_index);
                        if (m_update_book_url_queue == null || m_update_book_url_queue.Count <= 0)
                        {
                            t_page_index = 1;
                            GetCrawlerUrl(Consts.CrawlerType.UpdateBook, t_page_index);
                        }
                        t_page_index++;
                    }

                    if (m_important_book_url_queue == null || m_important_book_url_queue.Count <= 0)
                    {
                        GetCrawlerUrl(Consts.CrawlerType.ImportantBook, 0);


                    }
                }
                
                if (CrawlerType == Consts.CrawlerType.NewBook)
                {
             
                    if (m_new_book_url_queue == null || m_new_book_url_queue.Count <= 0)
                    {
                    
                        if (m_list != null && m_list.Count > 0)
                        {

                            foreach (BookSiteInfo t_site_info in m_list)
                            {
                                if (t_site_info.IsForbid)
                                    continue;

                                if (t_page_index >= t_site_info.EndPageIndex)
                                {
                                    LogHelper.Info(string.Format("{0}收录完成",t_site_info.SiteName));
                                    continue;
                                }

                                string t_book_list_url = t_site_info.BookListUrl;

                                SiteUrl t_site = new SiteUrl();
                                t_site.Url = string.Format(t_book_list_url, t_page_index);
                           
                                t_site.BookType = Consts.BookPageType.BookList;
                                t_site.SiteId = t_site_info.SiteId;
                                t_site.ChapterOrder = t_site_info.ChapterOrder;
                                t_site.SiteName = t_site_info.SiteName;
                                m_new_book_url_queue.Enqueue(t_site);
                            }


                        }

                        t_page_index++;
                    }
                  
                }
                if (CrawlerType == Consts.CrawlerType.UpdateBook)
                {
                    if (m_update_book_url_queue == null || m_update_book_url_queue.Count <= 0)
                    {
                        GetCrawlerUrl(Consts.CrawlerType.UpdateBook, t_page_index);
                        if (m_update_book_url_queue == null || m_update_book_url_queue.Count <= 0)
                        {
                            t_page_index = 1;
                            GetCrawlerUrl(Consts.CrawlerType.UpdateBook, t_page_index);
                        }
                        t_page_index++;
                    }
                }
                if (CrawlerType == Consts.CrawlerType.ImportantBook)
                {
                    if (m_important_book_url_queue == null || m_important_book_url_queue.Count <= 0)
                    {
                        GetCrawlerUrl(Consts.CrawlerType.ImportantBook, 0);

                        
                    }
                }

                Thread.Sleep( 1000);
            }
        }

        /// <summary>
        /// 收录新书
        /// </summary>
        private void StartNewBookCrawler()
        {
           
            while (true)
            {
                SiteUrl t_site = null;

                Monitor.Enter(m_new_book_url_queue);
                if (m_new_book_url_queue.Count > 0)
                {
                     t_site = m_new_book_url_queue.Dequeue();
                }
                Monitor.Exit(m_new_book_url_queue);

                if (t_site != null)
                {
                    StartUri(t_site, Consts.CrawlerType.NewBook);
                }

                Thread.Sleep(100);
                
            }
        }


        /// <summary>
        /// 收录更新小说
        /// </summary>
        private void StartUpdateBookCrawler()
        {
           
            while (true)
            {
                SiteUrl t_site = null;

                Monitor.Enter(m_update_book_url_queue);
                if (m_update_book_url_queue.Count > 0)
                {
                     t_site = m_update_book_url_queue.Dequeue();
                }
                Monitor.Exit(m_update_book_url_queue);

                if (t_site != null)
                {
                    StartUri(t_site, Consts.CrawlerType.UpdateBook);
                }

                Thread.Sleep(100);
           }
        }

        /// <summary>
        /// 收录重点小说
        /// </summary>
        private void StartImportantBook()
        {

            while (true)
            {

                SiteUrl t_site = null;

                Monitor.Enter(m_important_book_url_queue);
                if (m_important_book_url_queue.Count > 0)
                {
                     t_site = m_important_book_url_queue.Dequeue();
                }
                Monitor.Exit(m_important_book_url_queue);

                if (t_site != null)
                {
                    StartUri(t_site, Consts.CrawlerType.ImportantBook);
                }

                Thread.Sleep(100);
            
            }
        }



   

        /// <summary>
        /// 开始一个Url的收录
        /// </summary>
        /// <param name="i_uri"></param>
        /// <param name="i_request"></param>
        private void StartUri(SiteUrl i_site,Consts.CrawlerType i_crawler_type)
        {

            if (string.IsNullOrEmpty(i_site.Url))
                return;
            Uri t_uri=new Uri( i_site.Url);
          
            MyWebRequest t_request = null;

            BookSiteInfo t_book_site = null;
       
            foreach (BookSiteInfo t_book in m_list)
            {
                if (t_book.SiteId == i_site.SiteId)
                {
                    t_book_site = t_book;
                    break;
                }
            }

            //线程运行提示
            ThreadMsg t_thd_msg = new ThreadMsg();

            t_thd_msg.ThreadId = Thread.CurrentThread.Name.ToString();
            t_thd_msg.Action = "开始连接";
            t_thd_msg.ImageIndex = 1;
            t_thd_msg.BackColor = Color.WhiteSmoke;
            t_thd_msg.ForeColor = Color.Red;
            t_thd_msg.Url = t_uri.AbsoluteUri;

            SendMsg2Client(t_thd_msg, MsgType.Thread);

            string t_msg_conteng = "";

            if (t_request != null && t_request.m_response.m_keep_alive)
                t_msg_conteng += "Connection live to: " + t_uri.Host + "\r\n\r\n";
            else
                t_msg_conteng += "Connecting: " + t_uri.Host + "\r\n\r\n";

            try
            {
                t_request = MyWebRequest.Create(t_uri, t_request, false);
            
                t_request.m_timeout = m_sys_config.TimeOut * 1000;
                MyWebResponse t_response = t_request.GetResponse();
                
                t_msg_conteng += t_response.m_header;

                //请求提示
                RequestMsg t_request_msg = new RequestMsg();
                t_request_msg.OcurTime = DateTime.Now;
                t_request_msg.RequestHeader = t_uri.AbsoluteUri;
                t_request_msg.RequestBody = t_msg_conteng;

                SendMsg2Client(t_request_msg, MsgType.Request);

                //页面内容
                StringBuilder t_sb_page_content = new StringBuilder();
                byte[] t_page_bytes = new byte[10240];
                int t_bytes = 0;
                int t_total_bytes = 0;

                Encoding t_site_enconding = Encoding.GetEncoding(t_book_site.SiteEncoding);

                while ((t_bytes = t_response.m_socket.Receive(t_page_bytes, 0, 10240, SocketFlags.None)) > 0)
                {
                    t_total_bytes += t_bytes;
                    t_sb_page_content.Append(t_site_enconding.GetString(t_page_bytes, 0, t_bytes));


                    t_thd_msg = new ThreadMsg();

                    t_thd_msg.ThreadId = Thread.CurrentThread.Name.ToString();
                    t_thd_msg.Action = "获取页面内容";
                    t_thd_msg.ImageIndex = 1;
                    t_thd_msg.BackColor = Color.WhiteSmoke;
                    t_thd_msg.ForeColor = Color.Red;
                    t_thd_msg.Url = t_uri.AbsoluteUri;
                    t_thd_msg.ByteCount = t_total_bytes.ToString();
                    if (t_response.m_content_length != 0)
                    {
                        t_thd_msg.Percent = '%' + (100 - (t_response.m_content_length - t_total_bytes) * 100 / t_response.m_content_length).ToString();
                    }

                    SendMsg2Client(t_thd_msg, MsgType.Thread);
                }

                t_request_msg = new RequestMsg();
                t_request_msg.OcurTime = DateTime.Now;
                t_request_msg.RequestHeader = t_uri.AbsoluteUri;
                t_request_msg.RequestBody = "Connection closed.\r\n";

                SendMsg2Client(t_request_msg, MsgType.Request);

                string t_page_content = t_sb_page_content.ToString();

                t_thd_msg = new ThreadMsg();

                t_thd_msg.ThreadId = Thread.CurrentThread.Name.ToString();
                t_thd_msg.Action = "解析页面内容";
                t_thd_msg.ImageIndex = 1;
                t_thd_msg.BackColor = Color.WhiteSmoke;
                t_thd_msg.ForeColor = Color.Red;
                t_thd_msg.Url = t_uri.AbsoluteUri;
                t_thd_msg.ByteCount = t_total_bytes.ToString();
                t_thd_msg.Percent = "%100";

                SendMsg2Client(t_thd_msg, MsgType.Thread);

                t_thd_msg = new ThreadMsg();

                t_thd_msg.ThreadId = Thread.CurrentThread.Name.ToString();
                t_thd_msg.Action = "解析页面内容";
                t_thd_msg.ImageIndex = 1;
                t_thd_msg.BackColor = Color.WhiteSmoke;
                t_thd_msg.ForeColor = Color.Red;
                t_thd_msg.Url = t_uri.AbsoluteUri;
                t_thd_msg.ByteCount = t_total_bytes.ToString();
                t_thd_msg.Percent = "%100";

                //小说列表页 收录最新小说使用
                if (i_site.BookType == Consts.BookPageType.BookList)
                {
                    t_page_content = CrawlerUtil.ReplacePageHtml(t_page_content);
                    string t_book_name = string.Empty;
                    BookInfo t_book = null;
                    MatchCollection t_ma = Regex.Matches(t_page_content, t_book_site.BookListReg, RegexOptions.IgnoreCase | RegexOptions.Compiled);

                    Monitor.Enter(m_new_book_url_queue);

                    foreach (Match t_mc in t_ma)
                    {
                        if (string.IsNullOrEmpty(t_mc.Groups[t_book_site.BookUrlRegIndex].Value.ToString().Trim()))
                            continue;
                        t_book_name = t_mc.Groups[t_book_site.BookNameRegIndex].Value.ToString().Trim();
                        if (string.IsNullOrEmpty(t_book_name))
                            continue;
                        t_book = m_db_mgr.GetBookInfoByName(t_book_name);

                        SiteUrl t_book_url = new SiteUrl();
                        if (t_book != null)
                        {
                            LogHelper.Info("小说已经存在");
                            continue;
                            t_book_url.Url = CrawlerUtil.GetFullUrl(i_site.Url, t_book.ChapterListUrl.Trim());
                            t_book_url.BookId = t_book.BookId;
                            t_book_url.BookName = t_book.BookName;
                            t_book_url.SiteId = i_site.SiteId;
                            t_book_url.ChapterOrder = i_site.ChapterOrder;
                            t_book_url.LastUpdateChapterName = t_book.LastUpdateChapterName;
                            t_book_url.BookType = Consts.BookPageType.ChapterList;
                        }
                        else
                        {
                            t_book_url.Url = CrawlerUtil.GetFullUrl(i_site.Url, t_mc.Groups[t_book_site.BookUrlRegIndex].Value.ToString().Trim());
                            t_book_url.BookType = Consts.BookPageType.BookContent;
                            t_book_url.SiteId = t_book_site.SiteId;
                            t_book_url.SiteName = t_book_site.SiteName;
                            t_book_url.ChapterOrder = t_book_site.ChapterOrder;
                        }
                        m_new_book_url_queue.Enqueue(t_book_url);
                    }

                    Monitor.Exit(m_new_book_url_queue);

                    LogHelper.Info(string.Format("获取到{0}本小说",t_ma.Count));
                    t_thd_msg.Action = string.Format("获得{0}本小说", t_ma.Count);
                    SendMsg2Client(t_thd_msg, MsgType.Thread);
                }

                //小说介绍页
                if (i_site.BookType == Consts.BookPageType.BookContent)
                {
                    t_page_content = CrawlerUtil.ReplacePageHtml(t_page_content);

                    Match t_mc = Regex.Match(t_page_content, t_book_site.BookReg, RegexOptions.IgnoreCase | RegexOptions.Compiled);

                    if (t_mc != null)
                    {
                        BookInfo t_book = ParseBook(t_mc, t_book_site, i_site);

                        if (string.IsNullOrEmpty(t_book.BookName))
                        {
                            LogHelper.Info("未得到小说信息@!" + i_site.Url);
                            t_thd_msg.Action = "未得到小说信息";
                            SendMsg2Client(t_thd_msg, MsgType.Thread);
                            return;
                        }

                        t_thd_msg.Action = "获取到1个地址";
                        SendMsg2Client(t_thd_msg, MsgType.Thread);

                        long t_ret = m_db_mgr.AddBook(t_book);
                        if (t_ret > 0)
                        {
                            LogHelper.Info(string.Format("{0}入库成功", t_book.BookName));
                            t_thd_msg.Action = "【" + t_book.BookName + "】" + "入库成功";
                            SendMsg2Client(t_thd_msg, MsgType.Thread);
                        }
                        else
                        {
                            LogHelper.Info(string.Format("{0}已经存在", t_book.BookName));
                            t_thd_msg.Action = "【" + t_book.BookName + "】" + "已经存在";
                            SendMsg2Client(t_thd_msg, MsgType.Thread);
                        }

                        if (t_ret > 0)
                        {

                            SiteUrl t_book_url = new SiteUrl();
                            t_book_url.Url = CrawlerUtil.GetFullUrl(i_site.Url, t_book.ChapterListUrl.Trim());
                            t_book_url.BookId = t_ret;
                            t_book_url.BookName = t_book.BookName;
                            t_book_url.SiteId = i_site.SiteId;
                            t_book_url.ChapterOrder = i_site.ChapterOrder;
                            t_book_url.LastUpdateChapterName = t_book.LastUpdateChapterName;
                            t_book_url.BookType = Consts.BookPageType.ChapterList;

                            Monitor.Enter(m_new_book_url_queue);
                            m_new_book_url_queue.Enqueue(t_book_url);
                            Monitor.Exit(m_new_book_url_queue);
                        }
                    }
                    else
                    {
                        LogHelper.Info("没获取到小说信息" + i_site.Url);
                    }

                }

                //章节列表页
                if (i_site.BookType == Consts.BookPageType.ChapterList)
                {
                    if (i_site.BookId <= 0)
                    {
                        LogHelper.Info(string.Format("bookName={0},已经存在",i_site.BookName));
                        return;
                    }

                    t_page_content = CrawlerUtil.ReplacePageHtml(t_page_content);

                    MatchCollection t_ma = Regex.Matches(t_page_content, t_book_site.ChapterListReg, RegexOptions.IgnoreCase | RegexOptions.Compiled);

                    if (t_ma == null)
                        return;

                    ChapterInfo t_chpater=null;

                    int t_index = 0;

                    int t_total_count = t_ma.Count;

                    t_thd_msg.Action = string.Format("获取到{0}个地址", t_ma.Count);

                    SendMsg2Client(t_thd_msg, MsgType.Thread);

                    List<ChapterInfo> t_list_chapter = new List<ChapterInfo>();

                    DateTime t_update_time_now = DateTime.Now;

                    if (t_ma != null && t_ma.Count > 0)
                    {
                        t_update_time_now = DateTime.Now.AddSeconds(t_ma.Count * -4);
                    }

                    foreach (Match t_mc in t_ma)
                    {
                        t_chpater=new ChapterInfo();

                         if (t_mc.Groups.Count >= t_book_site.ChapterNameRegIndex)
                         {
                             t_chpater.ChapterName = t_mc.Groups[t_book_site.ChapterNameRegIndex].Value.ToString().Trim();
                         }

                         if (!string.IsNullOrEmpty(t_chpater.ChapterName) && t_chpater.ChapterName.IndexOf("名书楼")>-1)
                             continue;

                         if (t_mc.Groups.Count >= t_book_site.ChapterUrlRegIndex)
                         {
                             t_chpater.ChapterUrl =CrawlerUtil.GetFullUrl(i_site.Url, t_mc.Groups[t_book_site.ChapterUrlRegIndex].Value.ToString().Trim());
                         }

                         if (t_mc.Groups.Count >= t_book_site.UpdateTimeRegIndex && t_book_site.UpdateTimeRegIndex != -1)
                         {
                             DateTime t_update_time = DateTime.Now;
                             if (!DateTime.TryParse(t_mc.Groups[t_book_site.UpdateTimeRegIndex].Value.ToString(), out t_update_time))
                                 t_update_time = DateTime.Now;
                             t_chpater.UpdateTime = t_update_time;
                         }
                         else
                         {
                             t_index++;
                             //升序
                             if (i_site.ChapterOrder == ChapterOrderType.ASC)
                             {
                                 t_chpater.UpdateTime = t_update_time_now.AddSeconds(t_index * 3);
                             }
                             else if (i_site.ChapterOrder == ChapterOrderType.DESC)
                             {
                                 t_chpater.UpdateTime = t_update_time_now.AddSeconds(3*(t_total_count+5)).AddSeconds(t_index * (-3));
                             }
                         }

                         if (string.IsNullOrEmpty(t_chpater.ChapterName))
                             continue;

                         t_list_chapter.Add(t_chpater);

                    }

                    if (t_list_chapter.Count > 0)
                    {
                        t_list_chapter.Sort(delegate(ChapterInfo t_a, ChapterInfo t_b)
                        {
                            if (t_a.UpdateTime > t_b.UpdateTime)
                                return 1;
                            else if (t_a.UpdateTime == t_b.UpdateTime)
                                return 0;
                            return -1;
                        });
                    }

                    string t_table_name = "tbchapter";
                   

                    int t_insert_row_count = m_db_mgr.AddChapterList(i_site.BookId, i_site.BookName, t_list_chapter, t_table_name, i_site.SiteId, t_book_site.SiteName);


                    //下载章节内容
                 
                    //BaiDuChapter t_baidu_chapter = new BaiDuChapter();

                    //t_baidu_chapter.m_list = m_list;
                    //t_baidu_chapter.m_sys_config = m_sys_config;

                    //t_baidu_chapter.StartSEOChatper(i_site.BookId);
                   

                    //SendMsg2Client(t_thd_msg, MsgType.Thread);

                }



            }
            catch (Exception ex)
            {
                LogMsg t_log_msg = new LogMsg();
                t_log_msg.LogHeader = i_site.Url;
                t_log_msg.LogBody = "异常信息" + ex.ToString();
                t_log_msg.OcurTime = DateTime.Now;
                SendMsg2Client(t_log_msg, MsgType.Log);
                LogHelper.Error("收录url失败" + ex.ToString());
            }
        }

        /// <summary>
        /// 解析章节内容
        /// </summary>
        /// <param name="i_chapter_url"></param>
        /// <param name="i_page_content"></param>
        /// <param name="i_chapter_content_reg"></param>
        /// <param name="i_chaper_content_filter"></param>
        /// <returns></returns>
        private string GetChapterChapter(string i_chapter_url, string i_page_content, string i_chapter_content_reg, string i_chaper_content_filter,long i_book_id,long i_chapter_id)
        {
            string t_chapter_content = string.Empty;

            Match t_mc = Regex.Match(i_page_content, i_chapter_content_reg, RegexOptions.IgnoreCase | RegexOptions.Compiled);

            bool t_is_img = false;

            if (t_mc != null && t_mc.Groups.Count > 0)
            {
                t_chapter_content = t_mc.Groups[1].Value.ToString();

                if (!string.IsNullOrEmpty(i_chaper_content_filter))
                {
                    string[] t_filter_arr = i_chaper_content_filter.Split(new char[] { '$' }, StringSplitOptions.RemoveEmptyEntries);
                    foreach (string t_filter in t_filter_arr)
                    {
                        t_chapter_content = Regex.Replace(t_chapter_content, t_filter, "", RegexOptions.IgnoreCase | RegexOptions.Compiled);
                    }
                }

                //存在图片
                MatchCollection t_ma = Regex.Matches(t_chapter_content, "<img[^<>]*?src=s*[\'\"]*([^\"\']*)[\'\"]*[^<>]*?>", RegexOptions.IgnoreCase | RegexOptions.Compiled);

                foreach (Match t_mc_img in t_ma)
                {
                    string t_img_url = NetSiteCatchManager.GetFullUrl(i_chapter_url, t_mc_img.Groups[1].Value.ToString());
                    //下载图片
                    string t_file_name= NetSiteCatchManager.DownLoadFile(t_img_url, m_sys_config.BookIndexPath, i_book_id, i_chapter_id);

                    if (t_file_name == "noimg")
                    {
                        continue;
                    }
                    if (string.IsNullOrEmpty(t_file_name))
                    {
                        Uri t_ch = new Uri(i_chapter_url);
                        Uri t_img = new Uri(t_img_url);
                        if (t_ch.Host != t_img.Host)
                        {
                            t_chapter_content = string.Empty;
                            break;
                        }
                        t_file_name = t_img_url;
                    }
                   
                    t_chapter_content = Regex.Replace(t_chapter_content, t_mc_img.Groups[1].Value.ToString(),t_file_name);
                    t_is_img = true;
                }
                if (t_is_img == false)
                {
                    string t_temp_conent = Regex.Replace(t_chapter_content, "[【】（）a-zA-Z，！？(),!?;\\.；、……<>]", "", RegexOptions.IgnoreCase);
                    if (string.IsNullOrEmpty(t_temp_conent) || t_temp_conent.Length < 1000)
                        return string.Empty;
                }
                t_chapter_content = Regex.Replace(t_chapter_content, "<a[^<>]*?>.*?</a>", "", RegexOptions.IgnoreCase | RegexOptions.Compiled);

            }

            return t_chapter_content;

        }

        /// <summary>
        /// 解析小说信息
        /// </summary>
        /// <param name="i_mc"></param>
        /// <param name="i_book_site"></param>
        /// <returns></returns>
        private BookInfo ParseBook(Match i_mc,BookSiteInfo i_book_site,SiteUrl i_site)
        {
            try
            {
                BookInfo t_book = new BookInfo();
                t_book.BookName = i_mc.Groups[i_book_site.BookNameRegIndex2].Value.ToString().Trim().Replace("'","");
                t_book.AuthorName = i_mc.Groups[i_book_site.AuthorNameRegIndex].Value.ToString().Trim().Replace("'","");
                t_book.CategoryName = i_mc.Groups[i_book_site.CategoryNameRegIndex].Value.ToString().Trim().Replace("'","");
                t_book.SubCategoryName = i_mc.Groups[i_book_site.SubCategoryNameRegIndex].Value.ToString().Trim().Replace("'","");
                t_book.ICONUrl = CrawlerUtil.GetFullUrl(i_site.Url, i_mc.Groups[i_book_site.ICONRegIndex].Value.ToString().Trim()).Replace("'","");
                t_book.ChapterListUrl = CrawlerUtil.GetFullUrl(i_site.Url, i_mc.Groups[i_book_site.ChapterListUrlIndex].Value.ToString().Trim()).Replace("'","");
                t_book.Description = CrawlerUtil.FilterContent( i_mc.Groups[i_book_site.DescriptionRegIndex].Value.ToString().Trim());
                t_book.SiteId = i_site.SiteId;
                t_book.SiteName = i_site.SiteName;

                return t_book;
            }
            catch (Exception ex)
            {
                LogHelper.Error("解析小说介绍页失败" + ex.ToString());
                return null;
            }
        }

        #endregion

    }
}
